package com.searchdemo.webmagic;

import java.io.FileWriter;
import java.io.IOException;
import java.util.Map;

import org.apache.commons.io.IOUtils;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

public class JDgoodsProcesser implements PageProcessor {

	private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);

	@Override
	public void process(Page page) {
		Html all = page.getHtml();
		page.putField("all", all);
		page.putField("title", all.$("title"));
	/*	page.putField(
				"title",
				page.getHtml()
						.xpath("//*[@id='J_goodsList']/text()")
						.all());
		page.putField("content", page.getHtml().$("div.content").toString());
		page.putField("tags",
				page.getHtml().xpath("//*li[@class='gl-item']").all());*/
	}

	@Override
	public Site getSite() {
		return site;
	}

	public static void main(String[] args) {
		Pipeline pipeline = new Pipeline() {
			@Override
			public void process(ResultItems resultItems, Task task) {
				for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
					System.out.println(entry.getKey() + ":\t"
							+ entry.getValue());
					try {
						IOUtils.write(entry.getValue().toString(), new FileWriter("D://"+entry.getKey()+".html"));
					} catch (IOException e) {
						e.printStackTrace();
					}
				}
			}

		};
		Spider.create(new JDgoodsProcesser())
				.addUrl("https://search.jd.com/Search?keyword=%E5%8F%A3%E7%BD%A9&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&offset=2&page=1&s=1&click=0")
				.addPipeline(pipeline).run();
	}
}
